#!/usr/bin/env jruby
# -*- coding: us-ascii -*-
# Elizabeth Wiethoff 2008

require 'pp'
require 'enumerator'

########################################

class Symbol
  include Comparable
  def <=>(other)
    to_s <=> other.to_s
  end
end

########################################

class Paths < Hash
  def store(path)
    self[path.hash] = path
  end
end

########################################

# /usr/share/dict/connectives

class String
  def munge
    a = downcase.tr("'-", "").tr('^a-z', ' ').split
    # Now would be a good time to remove skip words and do synonym substitution.
    a.map { |word| word.intern }
  end
end

########################################

THRESHOLD = 3
DATA = Hash.new { |h, k| h[k] = Array.new }
PATHS = Paths.new

class Document
  def Document.process(path, text)
    PATHS.store path
    array = text.munge
    array.each_cons(THRESHOLD) { |a| DATA[a] << path.hash }
  end
end

########################################

Document.process('getty.txt', 'Four score and seven years ago...')
Document.process('yoda.txt', 'Score, and seven years you spend.')
Document.process('panda.txt', "You'll love the years you spend with pandas!")
Document.process('six.txt', 'Twenty-six and seven years make thirty-three.')
Document.process('bushel.txt', 
  'I love you a bushel and a peck, a bushel and a peck I do.')

pp DATA ; puts
pp PATHS ; puts

dupes = DATA.find_all { |k, v| v.size > 1 }
dupes.each { |k, v| v.map! { |f| PATHS[f] }.sort! }
DATA.clear ; PATHS.clear ; GC.start # :-)

pp dupes.sort_by { |k, v| [-v.size, v, k] } ; puts
